"""
爬取全国所有中小学学校名单
@date 2019/10/27
@author hujing
"""
import requests
from bs4 import BeautifulSoup


def get_city_uri():
    """
    获取所有城市的uri
    :return: 迭代器(uri，城市名称)
    """
    response = requests.get('http://www.xuexiaodaquan.com/')
    html = response.text.encode('iso-8859-1').decode('gbk')
    dom = BeautifulSoup(html, "html.parser")
    city_all_div = dom.find('div', 'city-all')
    a_list = city_all_div.find_all('a')
    # 拿到所有的城市链接
    for a in a_list:
        if a['href'] is '#':
            continue
        yield a['href'], a.get_text()


def get_all_school_name(find_url):
    """
    获取连接下的所有学校名称
    :param find_url: 访问链接
    :return:
    """
    response = requests.get(find_url)
    if response.status_code == 404:
        yield None
        return
    response.encoding = 'gbk'
    html = response.text
    dom = BeautifulSoup(html, "html.parser")
    content_div = dom.find(class_='list-xx clearfix')
    for dd in content_div.find_all('dd'):
        yield dd.find('a').getText()


def get_page_all_school_name(start_url):
    """
    获取所有学校名称， 包含翻页数据
    :param start_url:
    :return:
    """
    page = 1
    while True:
        is_finish = False
        # 第一页url不变
        if page > 1:
            this_url = start_url + 'pn' + str(page) + '.html'
        else:
            this_url = start_url
        for school_name in get_all_school_name(this_url):
            # 遍历结束
            if school_name is None:
                is_finish = True
                break
            yield school_name
        if is_finish:
            break
        page += 1


if __name__ == '__main__':
    for uri, city_name in get_city_uri():
        # 查询所有学校名称列表
        for school_type in ['youeryuan/', 'xiaoxue/', 'chuzhong/', 'gaozhong/']:
            url = uri + school_type
            for name in get_page_all_school_name(url):
                print(name)
